package com.chance.cc.crawler.development.bootstrap.xcar;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;

import java.util.HashMap;
import java.util.Map;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;
import static com.chance.cc.crawler.development.scripts.xcar.carseries.forum.XCarSeriesForumCrawlerScript.site;

/**
 * @author bx
 * @date 2021/1/11 0011 11:05
 */
public class XCarSeriesStart {

    public static final String domainId = "xcar";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {
     //  publicPraise();
        forumForLatestArticles();
      //  forumForLatestComments();

    }

    private static void publicPraise(){
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://newcar.xcar.com.cn/1437/")
                .releaseTime(System.currentTimeMillis())
                .needWashed(true)
                .needParsed(true)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .proxy(proxy)
                .build();
       // requestRecord.setDownload(false);
       // requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite("publicPraise");

        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,"public","praise","queue");
        CrawlerRequestRecord initNewsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xcar_news",turnPageItem)
                .httpUrl("http://192.168.1.215:9599/v1/meta/xcar/keys?site=carSeries")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .crawlerThreadNum(1)
                .triggerInfo(domainId, domainId, System.currentTimeMillis(), domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline()
                //.fileResultPipeline("D:\\chance\\log\\xcar\\xcar_praise.log", false)
                .requestRecord(requestRecord)
                //.supportRecord(initNewsCrawlerRecord)
                .build("com.chance.cc.crawler.development.scripts.xcar");
        devCrawlerController.start();
    }

    private static void forumForLatestArticles(){
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://newcar.xcar.com.cn/1917/")
                //.httpUrl("https://www.xcar.com.cn/bbs/xbbsapi/viewthread/getmore.php?tid=96954565&page=1&pid=1007873683&isReply=0&pageSize=10&order=2&_=1635991742130")
                .releaseTime(System.currentTimeMillis())
                .needWashed(true)
                .needParsed(true)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
//                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 6,new long[]{1632326400000l,System.currentTimeMillis()}))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24 * 6,null))
                .proxy(proxy)
                .build();
        //requestRecord.setDownload(false);
        //requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite("forum");
        Map<String,Object> map = new HashMap();
        map.put("forum_tid","aa");
        requestRecord.getHttpRequest().addExtra("forum_tid","97034507");
        requestRecord.getHttpRequest().addExtra("data_pid","1007873683");
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1635696000000l,System.currentTimeMillis()}));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));

        requestRecord.tagsCreator().bizTags().addCustomKV("custom_record_filter_info", JSON.toJSONString(filterCrawlerRecord));


        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,site,"latest","article","queue");
        CrawlerRequestRecord initNewsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xcar_news",turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/xcar/keys?site=bmw-kw")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .crawlerThreadNum(1)
                .triggerInfo(domainId, domainId, System.currentTimeMillis(), domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline()
                .fileResultPipeline("F:\\chance_log\\xcar\\forum\\xcar_forum_article.log", false)
                .requestRecord(requestRecord)
                //.supportRecord(initNewsCrawlerRecord)
                .build("com.chance.cc.crawler.development.scripts.xcar");
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag("order","dateline");
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        devCrawlerController.start();
    }

    private static void forumForLatestComments(){
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
//                .httpUrl("https://www.xcar.com.cn/bbs/xbbsapi/forumdisplay/get_thread_list.php")
                .httpUrl("https://www.xcar.com.cn/bbs/viewthread.php?tid=95855862")
                .httpHead("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36")
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(false)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*3,null))
//                .proxy(proxy)
                .build();

        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);
        requestRecord.tagsCreator().bizTags().addSite("forum");

        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(25,null));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));
        requestRecord.tagsCreator().bizTags().addCustomKV("custom_record_filter_info", JSON.toJSONString(filterCrawlerRecord));//文章过滤条件

        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,site,"latest","comment","queue");
        CrawlerRequestRecord initNewsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("xcar_news",turnPageItem)
                .httpUrl("http://192.168.1.215:9599/v1/meta/xcar/keys?site=carSeries")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController devCrawlerController = DevCrawlerController.builder()
                .crawlerThreadNum(1)
                .triggerInfo(domainId, domainId, System.currentTimeMillis(), domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline()
                .fileResultPipeline("D:\\chance\\log\\xcar\\xcar_comment_article.log", false)
                .requestRecord(requestRecord)
                .supportRecord(initNewsCrawlerRecord)
                .build();
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addKVTag("order","lastpost");
        devCrawlerController.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        devCrawlerController.start();
    }
}
